class AbstractFunction(ABC):
t = 0
@abstractmethod
def __call__(self, *args, **kwargs):
raise NotImplementedError
@abstractmethod
def derivative(self, *args, **kwargs):
raise NotImplementedError
@abstractmethod
def activate(self, *args, **kwargs):
raise NotImplementedError
class Neuron:
class Sigmoid(AbstractFunction):
t = 0.5
def __call__(self, x):
return 1.0 / (1.0 + np.exp(-x))
def derivative(self, x):
return self(x) * (1 - self(x))
def activate(self, x):
return np.where(self.__call__(x) >= self.t, 1, 0)
class Heaviside(AbstractFunction):
def __call__(self, x):
return np.heaviside(x, 1)
def derivative(self, x):
return np.ones_like(x)
def activate(self, x):
return np.where(self.__call__(x) > self.t, 1, 0)
class Relu(AbstractFunction):
t = 0.25
def __call__(self, x):
return x * (x > 0)
def derivative(self, x):
return (x > 0) * 1
def activate(self, x):
return np.where(self.__call__(x) > self.t, 1, 0)
class LRelu(AbstractFunction):
t = 0.25
def __init__(self, alpha=0.01):
self.alpha = alpha
def __call__(self, x):
return np.where(x > 0, x, x * 0.01)
def derivative(self, x):
return np.where(x > 0, 1, self.alpha)
def activate(self, x):
return np.where(self.__call__(x) >= self.t, 1, 0)
class Signum(AbstractFunction):
def __call__(self, x):
return np.sign(x)
def derivative(self, x):
return np.zeros_like(x)
def activate(self, x):
return np.where(self.__call__(x) > self.t, 1, 0)
class Sin(AbstractFunction):
def __call__(self, x):
return np.sin(x)
def derivative(self, x):
return np.cos(x)
def activate(self, x):
return np.where(self.__call__(x) >= self.t, 1, 0)
class Tanh(AbstractFunction):
def __call__(self, x):
return np.tanh(x)
def derivative(self, x):
return 1 / (np.cosh(x) ** 2)
def activate(self, x):
return np.where(self.__call__(x) >= self.t, 1, 0)
def __init__(self, activation_function, learning_rate, epochs, batchsize, dynamic_lr=False, min_lr=None,
max_lr=None) -> None:
self.coef_ = None
self.activation_function_name = activation_function
self.learning_rate = learning_rate
self.epochs = epochs
self.batchsize = batchsize
self.dynamic_lr = dynamic_lr
self.min_lr = min_lr
self.max_lr = max_lr
func_dict = {
'sigmoid': self.Sigmoid(),
'heaviside': self.Heaviside(),
'relu': self.Relu(),
'lrelu': self.LRelu(),
'signum': self.Signum(),
'sin': self.Sin(),
'tanh': self.Tanh()
}
self.activation_function = func_dict[activation_function]
pass
def fit(self, x, y):
# np.random.seed(seed)
if self.activation_function_name == 'signum' or self.activation_function_name == 'sin' or self.activation_function_name == 'tanh':
y[y == 0] = -1
y = y.reshape((-1, 1))
m, n = x.shape
self.coef_ = np.random.random(size=(1, x.shape[1] + 1))
X_with_bias = np.column_stack((np.ones(shape=(m, 1)) * -1, x))
batch = m - m % self.batchsize
for epoch in (range(self.epochs)):
if self.dynamic_lr:
self.learning_rate = self.min_lr + (self.max_lr - self.min_lr) * (1 + np.cos(epoch / self.epochs * np.pi))
indexes = np.arange(0, len(X_with_bias), 1)
np.random.shuffle(indexes)
X_trimmed = X_with_bias[indexes][:batch]
y_trimmed = y[indexes][:batch]
for i in range(0, batch, self.batchsize):
X_batch = X_trimmed[i:i + self.batchsize]
y_batch = y_trimmed[i:i + self.batchsize]
pred = self.activation_function(X_batch @ self.coef_.T)
dW = (y_batch - pred) * self.activation_function.derivative(X_batch @ self.coef_.T) * X_batch
self.coef_ += self.learning_rate * dW.mean(axis=0)
def predict(self, X) -> np.ndarray:
X = np.column_stack((np.ones(shape=(X.shape[0], 1)) * -1, X))
return self.activation_function.activate(X @ self.coef_.T)
def decision_function(self, X) -> np.ndarray:
X = np.column_stack((np.ones(shape=(X.shape[0], 1)) * -1, X))
return self.activation_function(X @ self.coef_.T)
def show_result(model, x_train, y_train, x_test, y_test, plot_title, show_Plot=True):
start = time.time()
model.fit(x_train, y_train)
y_pred = model.predict(x_test)
endtime = (time.time() - start)
fig, ax = plt.subplots(2, 2)
fig.set_size_inches((15, 15))
fig.suptitle(plot_title, fontsize=18)
if show_Plot:
mesh_data = c_[xx.ravel(), yy.ravel()]
Z = np.asarray(model.predict(mesh_data)).reshape(xx.shape)
ax[0][0].contourf(xx, yy, Z, alpha=0.6)
ax[0][0].scatter(x_test[:, 0], x_test[:, 1], c=y_test)
cm_model = confusion_matrix(y_test, y_pred)
tn_model = cm_model[0][0]
fn_model = cm_model[0][1]
tp_model = cm_model[1][1]
fp_model = cm_model[1][0]
ConfusionMatrixDisplay(cm_model).plot(ax=ax[0][1], cmap='binary', colorbar=False)
fpr, tpr, roc_auc = roc_curve(y_test, model.decision_function(x_test))
rok_model = RocCurveDisplay(fpr=fpr, tpr=tpr)
rok_model.plot(ax=ax[1][0], color='k')
labels = ('Sensitivity', 'Specificity', 'Iteration time')
y_pos = np.arange(len(labels))
values = [
tp_model / (tp_model + fn_model),
tn_model / (tn_model + fp_model),
endtime,
]
bars = ax[1][1].bar(y_pos, values, align='center', color='k')
ax[1][1].set_xticks(y_pos, labels=labels)
ax[1][1].bar_label(bars)
plt.show()
